import pytest
import time
import threading
from unittest.mock import Mock, patch, MagicMock
from requests.exceptions import Timeout, RequestException
from motor.resources.instance import Instance
from motor.resources.endpoint import Endpoint
from motor.coordinator.core.instance_manager import InstanceManager, UpdateInstanceMode
from motor.coordinator.core.instance_healthchecker import InstanceHealthChecker, ControllerClient


class TestInstanceHealthChecker:
    """InstanceHealthChecker unit test class"""

    def setup_method(self):
        """Reset singleton before each test"""
        # Completely reset the singleton instance
        if hasattr(InstanceHealthChecker, "_instance"):
            delattr(InstanceHealthChecker, "_instance")
        if hasattr(InstanceHealthChecker, "_initialized"):
            delattr(InstanceHealthChecker, "_initialized")
        
        # Also reset InstanceManager singleton if needed
        if hasattr(InstanceManager, "_instance"):
            delattr(InstanceManager, "_instance")

    @pytest.fixture
    def mock_instance(self):
        """Create mock Instance"""
        instance = Mock(spec=Instance)
        instance.id = 1
        instance.job_name = "test_job"
        instance.role = "P"
        instance.endpoints = {}
        instance.gathered_workload = Mock()
        return instance

    @pytest.fixture
    def mock_endpoint(self):
        """Create mock Endpoint"""
        endpoint = Mock(spec=Endpoint)
        endpoint.id = 1
        endpoint.ip = "127.0.0.1"
        endpoint.port = "8080"
        endpoint.device_infos = []
        endpoint.workload = Mock()
        return endpoint

    @pytest.fixture
    def mock_instance_manager(self):
        """Create mock InstanceManager"""
        manager = Mock(spec=InstanceManager)
        manager.update_instance_state = Mock()
        manager.delete_unavailable_instance = Mock()
        manager.is_available = Mock(return_value=True)
        return manager

    @pytest.fixture
    def mock_controller_client(self):
        """Create mock ControllerClient"""
        client = Mock(spec=ControllerClient)
        client.terminate_instance = Mock(return_value=True)
        client.report_alarm = Mock(return_value=True)
        return client

    @pytest.fixture
    def mock_config(self):
        """Create mock config"""
        config = Mock()
        config.max_consecutive_failures = 2
        config.dummy_request_interval = 0.1
        config.dummy_request_timeout = 5
        return config

    @pytest.fixture
    def health_checker(self, mock_controller_client, mock_instance_manager, mock_config):
        """Create health checker instance with fresh state"""
        # Create a fresh instance without going through the singleton initialization
        with patch('motor.coordinator.core.instance_healthchecker.CoordinatorConfig') as mock_config_class, \
             patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            
            mock_config_instance = Mock()
            mock_config_instance.health_check_config = mock_config
            mock_config_class.return_value.init.return_value = 0  # Simulate successful init
            mock_config_class.return_value.config = {'ID_gen_strategy': 'uuid'}  # Default config
            
            # Create instance directly, bypassing singleton complexity
            checker = InstanceHealthChecker.__new__(InstanceHealthChecker)
            
            # Manually initialize the instance with required attributes
            checker.controller_client = mock_controller_client
            checker.config = mock_config
            
            # Initialize the internal state that would be set in __init__
            checker._monitored_instances = {}
            checker._consecutive_failures = {}
            checker._is_running = False
            checker._monitoring_thread = None
            checker._lock = threading.RLock()
            checker._http_session = Mock()  # Mock the HTTP session
            
            # Mark as initialized to prevent reinitialization
            checker._initialized = True
            
            return checker

    def test_start_stop(self, health_checker):
        """Test start and stop functionality"""
        # Mock the monitoring thread to avoid actual thread creation
        mock_thread = Mock()
        mock_thread.is_alive.return_value = True
        
        # Use patch to mock threading.Thread
        with patch('threading.Thread') as mock_thread_class:
            # Configure the mock to return our mock_thread instance
            mock_thread_class.return_value = mock_thread
            
            # Test start
            health_checker.start()
            assert health_checker._is_running is True
            assert health_checker._monitoring_thread is mock_thread
            
            # Verify thread was created with correct parameters
            mock_thread_class.assert_called_once_with(
                target=health_checker._monitoring_loop,
                name="InstanceHealthChecker",
                daemon=True
            )
            
            # Verify thread was started
            mock_thread.start.assert_called_once()

        # Test duplicate start
        health_checker.start()  # Should not raise error

        # Test stop
        health_checker.stop()
        assert health_checker._is_running is False

    def test_push_exception_instance(self, health_checker, mock_instance, mock_endpoint, mock_instance_manager):
        """Test receiving abnormal instance"""
        # Push abnormal instance
        with patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            health_checker.push_exception_instance(mock_instance, mock_endpoint)

        # Verify instance was added to monitoring list
        assert mock_instance.id in health_checker._monitored_instances
        assert health_checker._consecutive_failures[mock_instance.id] == 0

        # Verify instanceManager was called
        mock_instance_manager.update_instance_state.assert_called_once_with(
            mock_instance.id, UpdateInstanceMode.UNAVAILABLE
        )

        # Test pushing same instance again
        with patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            health_checker.push_exception_instance(mock_instance, mock_endpoint)
        # Should not add duplicate - verify still only one call
        assert mock_instance_manager.update_instance_state.call_count == 1

    def test_check_state_alarm_available(self, health_checker, mock_instance_manager):
        """Test availability check (with available instances)"""
        with patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            result = health_checker.check_state_alarm()

        assert result is True
        mock_instance_manager.is_available.assert_called_once()

    def test_check_state_alarm_unavailable(self, health_checker, mock_instance_manager, mock_controller_client):
        """Test availability check (no available instances)"""
        mock_instance_manager.is_available.return_value = False

        with patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            result = health_checker.check_state_alarm()

        assert result is False
        mock_instance_manager.is_available.assert_called_once()
        mock_controller_client.report_alarm.assert_called_once_with(
            alarm_type="no_available_instances",
            message="No available P and D instances found",
            severity="critical"
        )

    def test_send_dummy_request_success(self, health_checker, mock_endpoint):
        """Test successful dummy request"""
        # Mock the HTTP session get method
        mock_response = Mock()
        mock_response.status_code = 200
        health_checker._http_session.get.return_value = mock_response

        result = health_checker._send_dummy_request(mock_endpoint)

        assert result is True
        health_checker._http_session.get.assert_called_once_with(
            f"http://{mock_endpoint.ip}:{mock_endpoint.port}/health",
            timeout=health_checker.config.dummy_request_timeout
        )

    def test_send_dummy_request_failure(self, health_checker, mock_endpoint):
        """Test failed dummy request"""
        # Mock the HTTP session get method to return error
        mock_response = Mock()
        mock_response.status_code = 500  # Server error
        health_checker._http_session.get.return_value = mock_response

        result = health_checker._send_dummy_request(mock_endpoint)

        assert result is False

    def test_send_dummy_request_timeout(self, health_checker, mock_endpoint):
        """Test dummy request timeout"""
        # Mock the HTTP session get method to raise timeout
        health_checker._http_session.get.side_effect = Timeout()

        result = health_checker._send_dummy_request(mock_endpoint)

        assert result is False

    def test_send_dummy_request_connection_error(self, health_checker, mock_endpoint):
        """Test dummy request connection error"""
        # Mock the HTTP session get method to raise connection error
        health_checker._http_session.get.side_effect = RequestException("Connection error")

        result = health_checker._send_dummy_request(mock_endpoint)

        assert result is False

    def test_send_dummy_request_missing_ip_port(self, health_checker, mock_endpoint):
        """Test dummy request with missing IP or port"""
        # Test missing IP
        mock_endpoint.ip = None
        result = health_checker._send_dummy_request(mock_endpoint)
        assert result is False

        # Test missing port
        mock_endpoint.ip = "127.0.0.1"
        mock_endpoint.port = None
        result = health_checker._send_dummy_request(mock_endpoint)
        assert result is False

    def test_recover_instance(self, health_checker, mock_instance, mock_endpoint, mock_instance_manager):
        """Test instance recovery"""
        # First add a monitored instance
        with patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            health_checker.push_exception_instance(mock_instance, mock_endpoint)
        
        # Reset mock call record
        mock_instance_manager.update_instance_state.reset_mock()

        # Recover instance
        with patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            health_checker._recover_instance(mock_instance.id)

        # Verify instance was recovered
        mock_instance_manager.update_instance_state.assert_called_once_with(
            mock_instance.id, UpdateInstanceMode.AVAILABLE
        )

        # Verify instance was removed from monitoring list
        assert mock_instance.id not in health_checker._monitored_instances

    def test_terminate_instance_success(self, health_checker, mock_instance, mock_endpoint, 
                                     mock_instance_manager, mock_controller_client):
        """Test successful instance termination"""
        # First add a monitored instance
        with patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            health_checker.push_exception_instance(mock_instance, mock_endpoint)
        
        # Manually set consecutive failures to reach threshold
        health_checker._consecutive_failures[mock_instance.id] = health_checker.config.max_consecutive_failures

        # Terminate instance
        with patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            health_checker._terminate_instance(mock_instance.id)

        # Verify controller was called
        mock_controller_client.terminate_instance.assert_called_once_with(
            instance_id=mock_instance.id,
            reason=f"Coordinator: detect instance {mock_instance.id} is abnormal"
        )

        # Verify instanceManager was called
        mock_instance_manager.delete_unavailable_instance.assert_called_once_with(mock_instance.id)

        # Verify instance was removed from monitoring list
        assert mock_instance.id not in health_checker._monitored_instances
        assert mock_instance.id not in health_checker._consecutive_failures

    def test_terminate_instance_controller_failure(self, health_checker, 
                                                mock_instance, mock_endpoint, mock_controller_client, mock_instance_manager):
        """Test instance termination when controller fails"""
        # Add monitored instance
        with patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            health_checker.push_exception_instance(mock_instance, mock_endpoint)
        
        # Manually set consecutive failures to reach threshold
        health_checker._consecutive_failures[mock_instance.id] = health_checker.config.max_consecutive_failures
        
        # Mock controller failure
        mock_controller_client.terminate_instance.return_value = False
        
        # Terminate instance
        with patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            health_checker._terminate_instance(mock_instance.id)
        
        # Verify controller was called but instance is still in monitoring due to failure
        mock_controller_client.terminate_instance.assert_called_once()
        assert mock_instance.id in health_checker._monitored_instances
        assert mock_instance.id in health_checker._consecutive_failures

    def test_check_single_instance_recovery(self, health_checker, mock_instance, mock_endpoint, mock_instance_manager):
        """Test instance recovery in single check"""
        # Add monitored instance
        with patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            health_checker.push_exception_instance(mock_instance, mock_endpoint)

        # Mock successful health check
        with patch.object(health_checker, '_send_dummy_request', return_value=True), \
             patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            # Execute single instance check
            health_checker._check_single_instance(mock_instance.id)

        # Verify instance was recovered
        mock_instance_manager.update_instance_state.assert_called_with(
            mock_instance.id, UpdateInstanceMode.AVAILABLE
        )
        assert mock_instance.id not in health_checker._monitored_instances

    def test_check_single_instance_termination(self, health_checker, mock_instance, mock_endpoint,
                                            mock_instance_manager, mock_controller_client):
        """Test instance termination in single check"""
        # Add monitored instance
        with patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            health_checker.push_exception_instance(mock_instance, mock_endpoint)

        # Set failure count to threshold - 1
        health_checker._consecutive_failures[mock_instance.id] = health_checker.config.max_consecutive_failures - 1

        # Mock failed health check
        with patch.object(health_checker, '_send_dummy_request', return_value=False), \
             patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            # Execute single instance check - should reach threshold and terminate
            health_checker._check_single_instance(mock_instance.id)

        # Verify instance was terminated
        mock_controller_client.terminate_instance.assert_called_once()
        mock_instance_manager.delete_unavailable_instance.assert_called_once_with(mock_instance.id)

        # Verify instance was removed from monitoring
        assert mock_instance.id not in health_checker._monitored_instances
        assert mock_instance.id not in health_checker._consecutive_failures

    def test_check_single_instance_increment_failure_count(self, health_checker, 
                                                       mock_instance, mock_endpoint, mock_instance_manager):
        """Test that failure count is incremented correctly"""
        # Add monitored instance
        with patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            health_checker.push_exception_instance(mock_instance, mock_endpoint)
        
        # Mock failed health check
        with patch.object(health_checker, '_send_dummy_request', return_value=False):
            # Execute check
            health_checker._check_single_instance(mock_instance.id)
            
            # Verify failure count was incremented
            assert health_checker._consecutive_failures[mock_instance.id] == 1

    def test_concurrent_access(self, health_checker):
        """Test concurrent access safety"""
        # Create instances and endpoints
        instances = []
        endpoints = []
        for i in range(10):
            instance = Mock(spec=Instance)
            instance.id = i
            instance.job_name = f"test_job_{i}"
            
            endpoint = Mock(spec=Endpoint)
            endpoint.id = i
            endpoint.ip = f"127.0.0.{i}"
            endpoint.port = "8080"
            
            instances.append(instance)
            endpoints.append(endpoint)

        # Simulate multiple concurrent operations
        threads = []
        for i in range(10):
            thread = threading.Thread(
                target=lambda: health_checker.push_exception_instance(instances[i], endpoints[i])
            )
            threads.append(thread)
            thread.start()

        # Wait for all threads to complete
        for thread in threads:
            thread.join()

        # Verify all instances were correctly added
        for i in range(10):
            assert i in health_checker._monitored_instances
            assert health_checker._consecutive_failures[i] == 0

    def test_error_handling_in_push_exception(self, health_checker, 
                                           mock_instance, mock_endpoint, mock_instance_manager):
        """Test error handling when pushing exception instance"""
        # Mock instanceManager throwing exception
        mock_instance_manager.update_instance_state.side_effect = Exception("Test error")

        # Should not raise exception, but log error
        with patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            health_checker.push_exception_instance(mock_instance, mock_endpoint)

        # Instance should still be added to monitoring list despite manager error
        assert mock_instance.id in health_checker._monitored_instances

    def test_check_single_instance_removed_during_check(self, health_checker, 
                                                     mock_instance, mock_endpoint,mock_instance_manager):
        """Test checking instance that gets removed during the check process"""
        # Add monitored instance
        with patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            health_checker.push_exception_instance(mock_instance, mock_endpoint)
        
        # Mock the scenario where instance is removed after getting endpoint but before sending request
        def mock_send_dummy_request(endpoint):
            # Remove instance during the health check
            with health_checker._lock:
                if mock_instance.id in health_checker._monitored_instances:
                    del health_checker._monitored_instances[mock_instance.id]
            return False
        
        with patch.object(health_checker, '_send_dummy_request', side_effect=mock_send_dummy_request):
            # This should not raise an exception
            health_checker._check_single_instance(mock_instance.id)
        
        # Instance should be removed
        assert mock_instance.id not in health_checker._monitored_instances

    def test_monitoring_loop_exception_handling(self, health_checker, 
                                               mock_instance, mock_endpoint, mock_instance_manager):
        """Test exception handling in monitoring loop"""
        # Add monitored instance
        with patch('motor.coordinator.core.instance_healthchecker.InstanceManager', return_value=mock_instance_manager):
            health_checker.push_exception_instance(mock_instance, mock_endpoint)
        
        # Mock _check_monitored_instances to raise exception
        with patch.object(health_checker, '_check_monitored_instances', side_effect=Exception("Test error")):
            # Should not raise exception, but continue loop
            health_checker._monitoring_loop()
        
        # Verify monitoring loop continues (no assertion needed, just shouldn't crash)

    def test_stop_without_start(self, health_checker):
        """Test stopping without starting"""
        # Should not raise any exception
        health_checker.stop()